import numpy as np
from collections import defaultdict
from location_.gsdmm import MovieGroupProcess
import re
import jieba
import csv
path = 'C:/Users/Administrator/Desktop/nj/noise_re.csv'
out_put = 'C:/Users/Administrator/Desktop/nj/noise_out.csv'
def clustering(order_loc):
    # folder_path = os.path.normcase(os.path.join(out_put, key_d))
    # if os.path.exists(folder_path):
    #     shutil.rmtree(folder_path)
    #     os.mkdir(folder_path)
    # else:
    #     os.mkdir(folder_path)

    mgp = MovieGroupProcess(K=150, alpha=0.5, beta=0.6, n_iters=40)
    labels = mgp.fit(docs=order_loc, vocab_size=60)
    return labels
def create_csv_for_all_class(key_d,labels, order_Narr,label_loc):
    num = np.unique(labels)
    class_orders = defaultdict(list)
    for i in range(num.shape[0]):
        label = num[i]
        # same_array_index = np.where(y_db == label)
        class_list = order_Narr[labels == label].tolist()
        class_orders[label] = class_list

    # for lb_key, cl_ord in class_orders.items:

    class_orders = dict(class_orders)
    for key, val in class_orders.items():
        district_1 = []
        k = str(int(key))
        for orde in val:
            district_1.append(orde[8])
        dis_set = list(set(district_1))
        for order_same_class in val:
            for dis in dis_set:
                if order_same_class[8] == dis:
                    new_key = ''.join([key_d,k, dis])
                    label_loc[new_key].append([order_same_class[i] for i in range(9)])
   
    return label_loc


orders = []
location = []
date_dict = defaultdict(list)

date = []
stop_words = []
district = []

with open('C:/Users/Administrator/Desktop/stop_words.csv','r') as stop_csv:
    readers = csv.reader(stop_csv)
    for r in readers:
        stop_words.append(r[0])
with open('C:/Users/Administrator/Desktop/1096_1.csv', 'r') as csvf:
        reader = csv.reader(csvf)
        for row in reader:
            orders.append(row)
            date.append(row[1])
            district.append(row[8])

date_set = list(set(date))
district_seted = list(set(district))
# label_loc = defaultdict(list)
order_noise = []
for TT in date_set:
    for order in orders:
        if order[1] == TT:
            date_dict[TT].append(order)
with open('C:/Users/Administrator/Desktop/noise_1.csv', 'w') as noise_csv:
    for key_d, values in date_dict.items():
        label_loc = defaultdict(list)
        order_loc = []
        order_N = []
        for ord in values:
            # for ord in values:
            adree_order = [ord[4], ord[4], ord[3]]
            link_adre = ''.join(adree_order)
            str_or = re.sub('\W+', '', link_adre)
            remove_order = re.sub('[a-zA-Z0-9_]', "", str_or)
            segs = jieba.cut(remove_order)
            addres = []
            adds = jieba.cut(ord[4])
            for add in adds:
                addres.append(add)
            address = ' '.join(fd for fd in addres)
            final = []
            for seg in segs:
                if seg not in stop_words:
                    final.append(seg)
            space_linked = ' '.join(f for f in final)
            aa = [address, address, address, space_linked]
            order_address = ' '.join(oa for oa in aa)
            if order_address:
                # linked_order = ''.join(f for f in final)
                order_N.append([ord[0], ord[1], ord[2], ord[3], ord[4], ord[5], ord[6], ord[7], ord[8]])
                order_loc.append(order_address)
            order_Narr = np.array(order_N)

        labels = clustering(order_loc)
        label_loc = create_csv_for_all_class(key_d, labels, order_Narr, label_loc)

    writer = csv.writer(noise_csv)

    for kk, vl in label_loc.items():
        for v in vl:
            v.append(kk)
            order_noise.append(v)
            writer.writerow(v)
    print(len(order_noise))

